# -*- coding: UTF-8 -*-
__author__ = 'tzq'
import time

from basetools import tool
from basetools.dbheper import *
from basetools.WebHelper import HttpHelper


class GetLianjiaData:
    def __init__(self):
        self.siteUrl = 'https://cd.lianjia.com/ershoufang/pg%s/'
        self.tool = tool.Tool()
        self.HttpHelper = HttpHelper()
        self.DBHelper = DBHelper()
        self.headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
            'Accept-Encoding': 'deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Cache-Control': 'max-age=0',
            'Connection': 'keep-alive',
            'Host': 'cd.lianjia.com',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'none',
            'Sec-Fetch-User': '?1',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36',
            'cache-control': 'no-cache',
        }

    def getContents(self, pageindex):
        contents = []
        url = self.siteUrl % str(pageindex)
        if pageindex == 1:
            url = self.siteUrl.replace('pg%s/', '')
        content = self.HttpHelper.getHtml(url, self.headers, False)
        if content is not None:
            import lxml.html.soupparser as soupparser
            dom = soupparser.fromstring(content)
            nodes = dom.xpath("//*[@class='bigImgList']/div[@class='item']")
            for item in nodes:
                try:
                    if len(item.xpath("./a[@class='img']")) > 0:

                        # 总价
                        total_price = item.xpath("./a[@class='img']/div[@class='price']/span")[0].text

                        # 类型  住宅/公寓/商业
                        resblockType = item.xpath("./div[@class='resblock-desc-wrapper']/div[@class='resblock-name']/span[@class='resblock-type']")[0].text

                        # 销售状态
                        saleStatus = item.xpath("./div[@class='resblock-desc-wrapper']/div[@class='resblock-name']/span[@class='sale-status']")[0].text

                        # 地址
                        resblockLocation = \
                            item.xpath("./div[@class='resblock-desc-wrapper']/div[@class='resblock-location']/a")[
                                0].text

                        # 面积
                        resblockArea = \
                            item.xpath("./div[@class='resblock-desc-wrapper']/div[@class='resblock-area']/span")[0].text

                        # 关键字
                        resblockTag = ""
                        tags = item.xpath("./div[@class='resblock-desc-wrapper']/div[@class='resblock-tag']/span")
                        for tag in tags:
                            resblockTag = resblockTag + "," + tag.text

                        # 价格
                        peice = item.xpath(
                            "./div[@class='resblock-desc-wrapper']/div[@class='resblock-price']/div[@class='main-price']/span[@class='number']")[
                            0].text

                        # 单位
                        peiceUnit = item.xpath(
                            "./div[@class='resblock-desc-wrapper']/div[@class='resblock-price']/div[@class='main-price']/span[@class='desc']")[
                            0].text

                        dict = {'resblockName': resblockName,
                                'resblockType': resblockType,
                                'saleStatus': saleStatus,
                                'resblockLocation': resblockLocation,
                                'resblockArea': resblockArea,
                                'resblockTag': resblockTag,
                                'peice': peice,
                                'peiceUnit': peiceUnit,
                                'sourceType': '链家'}
                        contents.append(dict)
                except:
                    print
        return contents

    def saveHouseInfo(self, pageIndex):
        # 获取第一页楼盘列表
        contents = self.getContents(pageIndex)
        if contents is not None:
            for item in contents:
                if item['resblockType'] == '住宅':
                    print(item)
                resblock_name = item['resblockName']
                resblock_type = item['resblockType']
                sale_status = item['saleStatus']
                resblock_location = item['resblockLocation']
                resblock_area = item['resblockArea']
                resblock_tag = item['resblockTag']
                peice = item['peice']
                peice_unit = item['peiceUnit']
                source_type = item['sourceType']
                self.DBHelper.InserHouseDatainfo(resblock_name, resblock_type, sale_status, resblock_location,
                                                 resblock_area, resblock_tag, float(peice), peice_unit, source_type, )

    def StartSpider(self, start, end):
        for i in range(start, end + 1):
            print(u"正在收集第", i, u"页的楼盘信息")
            self.saveHouseInfo(i)
            # time.sleep(10)


getLianjiaData = GetLianjiaData()
getLianjiaData.StartSpider(0, 100)
